package com.yc.testupload.crawler.test;

import com.yc.testupload.crawler.test.Question;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 牛客网爬虫类，用于爬取面试题数据
 */
public class NiukeSpider3 {
    // 设置爬虫时使用的Cookie
    private String cookies;

    // 模拟不同的用户代理，避免被反爬
    private static final String[] USER_AGENTS = {
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0"
    };

    private static final Random RANDOM = new Random();

    /**
     * 设置Cookie
     * @param cookies Cookie字符串
     */
    public void setCookies(String cookies) {
        this.cookies = cookies;
    }

    /**
     * 根据URL爬取面试题
     * @param url 面试题页面URL
     * @return 面试题列表
     */
    public List<Question> crawlQuestions(String url) {
        List<Question> questions = new ArrayList<>();
        try {
            // 初始化SSL上下文，信任所有证书
            initSSLContext();

            // 发送HTTP请求获取页面内容
            String htmlContent = getHtmlContent(url);
            if (htmlContent == null || htmlContent.isEmpty()) {
                System.out.println("获取页面内容失败");
                return questions;
            }

            // 打印调试信息，确认获取到了页面内容
            System.out.println("页面内容长度: " + htmlContent.length());
            
            // 先尝试从JavaScript状态对象中提取数据 - 增强版
            List<Question> jsStateQuestions = extractQuestionsFromJsState(htmlContent);
            if (!jsStateQuestions.isEmpty()) {
                System.out.println("成功从JS状态对象提取" + jsStateQuestions.size() + "道面试题");
                return jsStateQuestions;
            }

            // 如果从JS状态对象提取失败，使用JSoup解析HTML
            Document document = Jsoup.parse(htmlContent);

            // 尝试多种选择器策略来定位题目
            Elements questionElements = findQuestionElements(document);
            if (questionElements.isEmpty()) {
                System.out.println("未找到题目元素，尝试使用通用问答格式提取");
                // 如果没有找到结构化的题目元素，尝试使用通用问答格式提取
                return extractCommonQuestionAnswerFormat(htmlContent);
            }

            // 遍历所有题目元素并解析
            for (Element element : questionElements) {
                Question question = parseQuestion(element, document);
                if (question != null && !question.getContent().isEmpty() && 
                    (!question.getAnswer().isEmpty() || !question.getAnswer().contains("未找到"))) {
                    questions.add(question);
                }
            }

            // 如果提取的题目数量较少，尝试补充提取
            if (questions.size() < 5) {
                System.out.println("提取的题目数量较少，尝试补充提取");
                List<Question> supplementaryQuestions = extractCommonQuestionAnswerFormat(htmlContent);
                for (Question q : supplementaryQuestions) {
                    if (!questions.contains(q) && !q.getContent().isEmpty()) {
                        questions.add(q);
                    }
                }
            }

            System.out.println("成功爬取" + questions.size() + "道面试题");
        } catch (Exception e) {
            System.out.println("爬取过程中发生错误: " + e.getMessage());
            e.printStackTrace();
        }
        return questions;
    }

    /**
     * 从JavaScript状态对象中提取题目数据 - 增强版，特别针对牛客网页面
     */
    private List<Question> extractQuestionsFromJsState(String htmlContent) {
        List<Question> questions = new ArrayList<>();
        try {
            // 1. 尝试从window.__INITIAL_STATE__提取数据
            Pattern statePattern = Pattern.compile("window\\.__INITIAL_STATE__\\s*=\\s*(.*?);\\s*</script>", Pattern.DOTALL);
            Matcher stateMatcher = statePattern.matcher(htmlContent);
            
            if (stateMatcher.find()) {
                String jsonData = stateMatcher.group(1);
                System.out.println("找到INITIAL_STATE数据");
                
                // 尝试从JSON数据中提取题目
                return extractQuestionsFromJsonData(jsonData);
            }
            
            // 2. 尝试从Vue状态对象提取数据
            // 使用StringBuilder构建正则表达式，确保所有转义字符正确
            StringBuilder patternBuilder = new StringBuilder();
            patternBuilder.append("new\\\\s+Vue\\\\s*\\\\(\\\\{"); // new Vue ({
            patternBuilder.append(".*?"); // 任意字符，非贪婪
            patternBuilder.append("data\\\\s*:\\s*"); // data: 
            patternBuilder.append("([^}]*)\\}"); // 捕获除了}以外的所有字符和结束的}
            
            Pattern vuePattern = Pattern.compile(patternBuilder.toString());
            Matcher vueMatcher = vuePattern.matcher(htmlContent);
            
            if (vueMatcher.find()) {
                String jsonData = vueMatcher.group(1);
                System.out.println("找到Vue数据");
                
                // 尝试从JSON数据中提取题目
                return extractQuestionsFromJsonData(jsonData);
            }
            
            // 3. 尝试从牛客网特定的数据结构提取
            Pattern niukeDataPattern = Pattern.compile("data-questions=\\\"(.*?)\\\"", Pattern.DOTALL);
            Matcher niukeDataMatcher = niukeDataPattern.matcher(htmlContent);
            
            if (niukeDataMatcher.find()) {
                String jsonData = niukeDataMatcher.group(1);
                System.out.println("找到牛客网特定数据结构");
                
                // 尝试从JSON数据中提取题目
                return extractQuestionsFromJsonData(jsonData);
            }
            
            // 4. 尝试从window.questions提取数据
            Pattern windowQuestionsPattern = Pattern.compile("window\\.questions\\s*=\\s*(.*?);\\s*</script>", Pattern.DOTALL);
            Matcher windowQuestionsMatcher = windowQuestionsPattern.matcher(htmlContent);
            
            if (windowQuestionsMatcher.find()) {
                String jsonData = windowQuestionsMatcher.group(1);
                System.out.println("找到window.questions数据");
                
                // 尝试从JSON数据中提取题目
                return extractQuestionsFromJsonData(jsonData);
            }
            
            // 5. 尝试从window.paperData提取数据
            Pattern paperDataPattern = Pattern.compile("window\\.paperData\\s*=\\s*(.*?);\\s*</script>", Pattern.DOTALL);
            Matcher paperDataMatcher = paperDataPattern.matcher(htmlContent);
            
            if (paperDataMatcher.find()) {
                String jsonData = paperDataMatcher.group(1);
                System.out.println("找到window.paperData数据");
                
                // 尝试从JSON数据中提取题目
                return extractQuestionsFromJsonData(jsonData);
            }
        } catch (Exception e) {
            System.out.println("从JS状态对象提取数据时发生错误: " + e.getMessage());
        }
        return questions;
    }
    
    /**
     * 从JSON数据中提取题目 - 增强版
     */
    private List<Question> extractQuestionsFromJsonData(String jsonData) {
        List<Question> questions = new ArrayList<>();
        try {
            // 清理JSON数据，移除不必要的转义字符
            jsonData = jsonData.replaceAll("\\\\", "\\");
            jsonData = jsonData.replaceAll("\\'", "'");
            jsonData = jsonData.replaceAll("\\\"", "\"");
            
            // 尝试提取数据的不同路径 - 增加更多牛客网可能的数据路径
            String[][] paths = {
                {"prefetchData", "paperDetail"}, 
                {"questionList"}, 
                {"questions"},
                {"data", "list"}, 
                {"quizList"}, 
                {"examPaper"}, 
                {"paperData"},
                {"data", "questions"},
                {"data", "examQuestions"},
                {"result", "data", "questions"},
                {"data", "content", "questions"},
                {"examContent", "questions"},
                {"testPaper", "questions"}
            };
            
            for (String[] path : paths) {
                String searchPath = buildSearchPath(path);
                Pattern itemPattern = Pattern.compile(
                    searchPath + "\\s*:\\s*\\[([^\\]]*?)\\]", 
                    Pattern.DOTALL
                );
                Matcher itemMatcher = itemPattern.matcher(jsonData);
                
                if (itemMatcher.find()) {
                    String itemsData = itemMatcher.group(1);
                    questions.addAll(extractQuestionsFromItemsData(itemsData));
                    if (!questions.isEmpty()) {
                        break;
                    }
                }
            }
            
            // 如果上面的方法都没找到，直接从JSON中提取题目文本
            if (questions.isEmpty()) {
                questions.addAll(extractQuestionsFromRawJson(jsonData));
            }
        } catch (Exception e) {
            System.out.println("从JSON数据提取题目时发生错误: " + e.getMessage());
        }
        return questions;
    }
    
    /**
     * 构建JSON搜索路径 - 优化版
     */
    private String buildSearchPath(String[] pathParts) {
        StringBuilder path = new StringBuilder();
        for (int i = 0; i < pathParts.length; i++) {
            if (i > 0) path.append("\\.\\s*");
            path.append("(?:\"\\s*:\\s*").append(pathParts[i]).append("\\s*").append("|'\\s*:\\s*").append(pathParts[i]).append("\\s*").append("|\\s*:\\s*").append(pathParts[i]).append(")");
        }
        return path.toString();
    }
    
    /**
     * 从题目项数据中提取题目 - 增强版
     */
    private List<Question> extractQuestionsFromItemsData(String itemsData) {
        List<Question> questions = new ArrayList<>();
        
        // 分割题目项
        Pattern itemPattern = Pattern.compile("\\{[^}]*?\\}(?:,|$)");
        Matcher itemMatcher = itemPattern.matcher(itemsData);
        
        while (itemMatcher.find() && questions.size() < 30) {
            String itemJson = itemMatcher.group().replaceAll(",$", "");
            Question question = extractQuestionFromItemJson(itemJson);
            if (question != null && question.getContent() != null && !question.getContent().isEmpty()) {
                questions.add(question);
            }
        }
        
        return questions;
    }
    
    /**
     * 从单个题目JSON中提取题目 - 增强版
     */
    private Question extractQuestionFromItemJson(String itemJson) {
        Question question = new Question();
        
        try {
            // 提取题目ID/编号
            String number = extractFieldFromJson(itemJson, "id", "questionId", "number", "index");
            if (number != null) {
                question.setNumber(number);
            } else {
                // 如果没有找到编号，使用问题列表中的位置
                question.setNumber("0");
            }
            
            // 提取题目标题
            String title = extractFieldFromJson(itemJson, "title", "name", "stem", "questionName");
            if (title != null) {
                question.setTitle(title);
            }
            
            // 提取题目内容
            String content = extractFieldFromJson(itemJson, "content", "stem", "question", "description");
            if (content != null) {
                // 清理HTML标签
                content = cleanHtml(content);
                question.setContent(content);
                
                // 如果没有标题，使用内容的前60个字符作为标题
                if (title == null) {
                    question.setTitle(content.length() > 60 ? content.substring(0, 60) + "..." : content);
                }
            }
            
            // 提取答案/解析 - 增强版，尝试更多可能的字段
            String answer = extractFieldFromJson(itemJson, "answer", "analysis", "explanation", 
                "referenceAnswer", "correctAnswer", "rightAnswer", "detail", "solution");
            if (answer != null) {
                // 清理HTML标签
                answer = cleanHtml(answer);
                question.setAnswer(answer);
            }
            
            // 如果没有找到答案，尝试查找选项和正确答案
            if (answer == null) {
                StringBuilder optionsBuilder = new StringBuilder();
                
                // 尝试从options字段提取
                Pattern optionsArrayPattern = Pattern.compile("(?:\"|')options(?:\"|')\\s*:\\s*\\[([^\\]]*)\\]");
                Matcher optionsArrayMatcher = optionsArrayPattern.matcher(itemJson);
                
                if (optionsArrayMatcher.find()) {
                    String optionsArray = optionsArrayMatcher.group(1);
                    Pattern optionPattern = Pattern.compile("\\{[^}]*?\\}");
                    Matcher optionMatcher = optionPattern.matcher(optionsArray);
                    
                    while (optionMatcher.find()) {
                        String optionJson = optionMatcher.group();
                        String optionText = extractFieldFromJson(optionJson, "content", "text", "value");
                        String optionLabel = extractFieldFromJson(optionJson, "label", "optionId");
                        
                        if (optionText != null) {
                            // 避免使用正则表达式，使用字符串替换方法删除引号
                            optionText = cleanHtml(optionText)
                                .replace("\"", "")
                                .replace("'", "")
                                .trim();
                            if (optionLabel != null) {
                                optionsBuilder.append(optionLabel).append(": ").append(optionText).append("\n");
                            } else {
                                optionsBuilder.append(optionText).append("\n");
                            }
                        }
                    }
                }
                
                // 尝试直接提取option字段 - 使用简化的方法避免复杂转义
                if (optionsBuilder.length() == 0) {
                    // 寻找所有可能的option字段，如"option": "value" 或 "optionA": "value"等
                    String[] optionKeys = {"option", "optionA", "optionB", "optionC", "optionD"};
                    for (String key : optionKeys) {
                        String optionValue = extractFieldFromJson(itemJson, key);
                        if (optionValue != null && !optionValue.isEmpty()) {
                            optionsBuilder.append(key.substring(6)).append(": ").append(optionValue).append("\n");
                        }
                    }
                }
                
                String options = optionsBuilder.toString().trim();
                if (!options.isEmpty()) {
                    // 添加到题目内容中
                    question.setContent(question.getContent() + "\n\n选项：\n" + options);
                }
                
                // 查找正确答案
                String correctAnswer = extractFieldFromJson(itemJson, "correctAnswer", "rightAnswer", 
                    "answer", "correctOption", "rightOption", "answerKey");
                if (correctAnswer != null) {
                    question.setAnswer("正确答案：" + cleanHtml(correctAnswer));
                }
            }
            
        } catch (Exception e) {
            System.out.println("解析单个题目JSON时发生错误: " + e.getMessage());
            return null;
        }
        
        return question;
    }
    
    /**
     * 从JSON字符串中提取指定字段的值 - 简化版，避免复杂转义
     */
    private String extractFieldFromJson(String json, String... fieldNames) {
        if (json == null || json.isEmpty()) {
            return null;
        }
        
        for (String fieldName : fieldNames) {
            try {
                // 构建简化的模式，只匹配基本的字段格式，避免复杂转义
                String searchStart = "\"" + fieldName + "\":";
                int pos = json.toLowerCase().indexOf(searchStart.toLowerCase());
                
                if (pos != -1) {
                    // 跳过字段名，定位到值的开始
                    pos += searchStart.length();
                    
                    // 跳过空白字符
                    while (pos < json.length() && Character.isWhitespace(json.charAt(pos))) {
                        pos++;
                    }
                    
                    // 提取值
                    if (pos < json.length()) {
                        if (json.charAt(pos) == '"') {
                            // 处理字符串值（带引号）
                            int endPos = json.indexOf('"', pos + 1);
                            while (endPos != -1 && endPos > 0 && json.charAt(endPos - 1) == '\\') {
                                endPos = json.indexOf('"', endPos + 1);
                            }
                            if (endPos != -1) {
                                return cleanHtml(json.substring(pos + 1, endPos));
                            }
                        } else {
                            // 处理非字符串值
                            int endPos = json.indexOf(',', pos);
                            if (endPos == -1 || endPos > json.indexOf('}', pos)) {
                                endPos = json.indexOf('}', pos);
                            }
                            if (endPos != -1) {
                                return cleanHtml(json.substring(pos, endPos).trim());
                            }
                        }
                    }
                }
            } catch (Exception e) {
                // 发生异常时记录错误并继续尝试其他字段
                System.out.println("提取字段" + fieldName + "时发生错误: " + e.getMessage());
            }
        }
        return null;
    }
    
    /**
     * 从原始JSON中提取题目文本 - 增强版
     */
    private List<Question> extractQuestionsFromRawJson(String jsonData) {
        List<Question> questions = new ArrayList<>();
        
        try {
            // 清理JSON中的转义字符
            String cleanJson = jsonData.replace("\\", "");
            
            // 提取包含题目关键词的文本块 - 增强版，增加更多关键词
            String[] keywords = {"题目", "问题", "题干", "QUESTION", "Problem", "问答", "问答对", 
                "Q:", "问:", "问题:", "题目:", "问答题:", "选择题:", "判断题:", "编程题:", "简答题:"};
            StringBuilder allText = new StringBuilder();
            
            for (String keyword : keywords) {
                Pattern textPattern = Pattern.compile(".*?" + keyword + ".*?", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
                Matcher textMatcher = textPattern.matcher(cleanJson);
                
                while (textMatcher.find()) {
                    allText.append(textMatcher.group()).append("\n");
                }
            }
            
            String combinedText = cleanHtml(allText.toString());
            
            // 按行分割文本
            String[] lines = combinedText.split("\\n+");
            StringBuilder currentQuestion = new StringBuilder();
            StringBuilder currentAnswer = new StringBuilder();
            boolean inQuestion = false;
            int count = 0;
            
            for (String line : lines) {
                line = line.trim();
                if (line.isEmpty() || line.length() < 5) continue;
                
                // 检查是否是题目开始 - 增强版，增加更多匹配模式
                if (line.matches("问题\\d+[：:.]?.*") || 
                    line.matches("\\d+[、.]\\s*") || 
                    line.matches("题目\\d+[：:.]?.*") ||
                    line.contains("题干") && !line.contains("答案") && !line.contains("解析") ||
                    line.matches("Q:[\\s]*.*") ||
                    line.matches("问:[\\s]*.*") ||
                    line.matches("选择题[\\s]*\\d+") ||
                    line.matches("编程题[\\s]*\\d+") ||
                    line.matches("简答题[\\s]*\\d+")) {
                    
                    // 如果已经在处理题目，先保存之前的题目
                    if (inQuestion && currentQuestion.length() > 0 && count > 0) {
                        Question q = new Question();
                        q.setNumber(String.valueOf(count));
                        q.setTitle(extractQuestionTitle(currentQuestion.toString()));
                        q.setContent(currentQuestion.toString().trim());
                        q.setAnswer(currentAnswer.toString().trim());
                        questions.add(q);
                        currentAnswer.setLength(0);
                    }
                    
                    // 开始新的题目
                    currentQuestion.setLength(0);
                    currentQuestion.append(line);
                    inQuestion = true;
                    count++;
                }
                // 检查是否是答案开始 - 增强版，增加更多匹配模式
                else if (inQuestion && (line.contains("答案") || 
                                         line.contains("解析") ||
                                         line.contains("详解") ||
                                         line.contains("正确选项") ||
                                         line.contains("解题思路") ||
                                         line.matches("A:[\\s]*.*") && currentQuestion.length() > 0 ||
                                         line.matches("B:[\\s]*.*") && currentQuestion.length() > 0 ||
                                         line.matches("C:[\\s]*.*") && currentQuestion.length() > 0 ||
                                         line.matches("D:[\\s]*.*") && currentQuestion.length() > 0)) {
                    currentAnswer.append(line);
                }
                // 属于当前问题或答案的内容
                else if (inQuestion) {
                    if (line.contains("答案") || 
                        line.contains("解析") ||
                        line.contains("详解") ||
                        line.contains("正确选项") ||
                        line.contains("解题思路")) {
                        currentAnswer.append("\n").append(line);
                    } else {
                        currentQuestion.append("\n").append(line);
                    }
                }
            }
            
            // 保存最后一个题目
            if (inQuestion && currentQuestion.length() > 0 && count > 0) {
                Question q = new Question();
                q.setNumber(String.valueOf(count));
                q.setTitle(extractQuestionTitle(currentQuestion.toString()));
                q.setContent(currentQuestion.toString().trim());
                q.setAnswer(currentAnswer.toString().trim());
                questions.add(q);
            }
            
        } catch (Exception e) {
            System.out.println("从原始JSON提取题目时发生错误: " + e.getMessage());
        }
        
        return questions;
    }
    
    /**
     * 使用通用问答格式提取题目 - 增强版
     */
    private List<Question> extractCommonQuestionAnswerFormat(String htmlContent) {
        List<Question> questions = new ArrayList<>();
        Set<String> uniqueQuestionIds = new HashSet<>(); // 用于去重的集合
        
        try {
            // 移除HTML标签，减少处理的复杂度
            String text = cleanHtml(htmlContent);
            
            // 首先尝试使用段落分割法，这通常更可靠
            System.out.println("使用段落分割法提取题目");
            
            // 使用段落分割
            String[] paragraphs = text.split("\n{2,}");
            StringBuilder questionBuilder = null;
            boolean inQuestion = false;
            Map<Integer, String> tempQuestions = new HashMap<>(); // 临时存储题目内容，按序号组织
            
            for (String paragraph : paragraphs) {
                paragraph = paragraph.trim();
                if (paragraph.isEmpty() || paragraph.length() < 10) continue; // 跳过太短的段落
                
                // 检查是否是题目的开始
                if (paragraph.matches("^问题\\d+[：:.]?.*") || 
                    paragraph.matches("^\\d+[、.]\\s+.+") || 
                    paragraph.matches("^题目\\d+[：:.]?.*") ||
                    paragraph.contains("题干") && !paragraph.contains("答案") && !paragraph.contains("解析") ||
                    paragraph.matches("^Q:[\\s]*.+") ||
                    paragraph.matches("^问:[\\s]*.+") ||
                    paragraph.matches("^选择题[\\s]*\\d+") ||
                    paragraph.matches("^编程题[\\s]*\\d+") ||
                    paragraph.matches("^简答题[\\s]*\\d+")) {
                        
                    // 如果已经在处理题目，先保存之前的题目
                    if (inQuestion && questionBuilder != null && questionBuilder.length() > 20) { // 只保存有一定长度的题目
                        String content = questionBuilder.toString().trim();
                        String number = extractQuestionNumber(content);
                        if (number != null && !number.equals("0")) { // 跳过编号为0的题目
                            try {
                                int num = Integer.parseInt(number);
                                tempQuestions.put(num, content); // 按序号存储，自动覆盖重复序号
                            } catch (NumberFormatException e) {
                                // 如果序号不是数字，使用内容的哈希码作为临时序号
                                tempQuestions.put(content.hashCode(), content);
                            }
                        }
                    }
                    
                    // 开始新的题目
                    questionBuilder = new StringBuilder(paragraph);
                    inQuestion = true;
                } else if (inQuestion && questionBuilder != null) {
                    // 继续当前题目
                    questionBuilder.append("\n").append(paragraph);
                }
            }
            
            // 保存最后一个题目
            if (inQuestion && questionBuilder != null && questionBuilder.length() > 20) {
                String content = questionBuilder.toString().trim();
                String number = extractQuestionNumber(content);
                if (number != null && !number.equals("0")) {
                    try {
                        int num = Integer.parseInt(number);
                        tempQuestions.put(num, content);
                    } catch (NumberFormatException e) {
                        tempQuestions.put(content.hashCode(), content);
                    }
                }
            }
            
            // 如果段落分割法没有找到足够的题目，尝试使用正则表达式方法
            if (tempQuestions.isEmpty()) {
                System.out.println("段落分割法未找到题目，尝试使用正则表达式方法");
                // 使用正则表达式提取所有以数字序号开始的文本块
                Pattern numberedPattern = Pattern.compile("(\\d+[、.])\\s+([^\\n]+(?:\\n[^\\d]+)*)", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
                Matcher numberedMatcher = numberedPattern.matcher(text);
                
                while (numberedMatcher.find()) {
                    String questionBlock = numberedMatcher.group(0).trim();
                    String number = numberedMatcher.group(1).replaceAll("[、.]", "").trim(); // 提取并清理序号
                    
                    if (number != null && !number.equals("0")) {
                        try {
                            int num = Integer.parseInt(number);
                            tempQuestions.put(num, questionBlock);
                        } catch (NumberFormatException e) {
                            tempQuestions.put(questionBlock.hashCode(), questionBlock);
                        }
                    }
                }
            }
            
            // 将临时存储的题目转换为Question对象，并按序号排序
            List<Integer> sortedNumbers = new ArrayList<>(tempQuestions.keySet());
            Collections.sort(sortedNumbers);
            
            // 如果按序号排序后数量太少，尝试使用自然顺序
            if (sortedNumbers.size() < 2) {
                sortedNumbers = new ArrayList<>(tempQuestions.keySet());
            }
            
            for (Integer key : sortedNumbers) {
                if (questions.size() >= 30) break; // 限制最多提取30道题
                
                String questionBlock = tempQuestions.get(key);
                String number = extractQuestionNumber(questionBlock);
                if (number == null || number.isEmpty()) {
                    number = String.valueOf(key); // 使用临时序号
                }
                
                // 尝试查找答案部分 - 增强版，增加更多关键词
                int answerStart = -1;
                String[] answerKeywords = {"答案", "解析", "正确选项", "解题思路", "详解", "A:", "B:", "C:", "D:", "参考答案", "正确答案", "正确选项是"};
                for (String keyword : answerKeywords) {
                    int pos = questionBlock.indexOf(keyword);
                    if (pos != -1 && pos > 10) { // 确保不在题目开头
                        if (answerStart == -1 || pos < answerStart) {
                            answerStart = pos;
                        }
                    }
                }
                
                // 提取题目内容和答案
                String content = questionBlock;
                String answerText = "";
                
                if (answerStart != -1) {
                    content = questionBlock.substring(0, answerStart).trim();
                    answerText = questionBlock.substring(answerStart).trim();
                }
                
                // 创建题目对象
                Question question = new Question();
                question.setNumber(number);
                question.setTitle(extractQuestionTitle(content));
                question.setContent(content);
                question.setAnswer(answerText.isEmpty() ? "答案未找到，请手动添加" : answerText);
                
                // 添加题目，并避免重复
                String questionId = number + ":" + content.hashCode(); // 使用序号和内容哈希作为唯一标识
                if (!uniqueQuestionIds.contains(questionId)) {
                    uniqueQuestionIds.add(questionId);
                    questions.add(question);
                }
            }
            
            // 如果提取的题目数量太少，尝试直接从文本中提取关键词
            if (questions.isEmpty()) {
                System.out.println("常规方法未找到题目，尝试使用关键词提取法");
                extractQuestionsWithKeywords(text, questions, uniqueQuestionIds);
            }
            
        } catch (Exception e) {
            System.out.println("使用通用问答格式提取题目时发生错误: " + e.getMessage());
            e.printStackTrace();
        }
        
        return questions;
    }
    
    /**
     * 使用关键词提取法提取题目
     */
    private void extractQuestionsWithKeywords(String text, List<Question> questions, Set<String> uniqueQuestionIds) {
        try {
            // 分割文本为句子
            String[] sentences = text.split("[。！？；；.!?;]");
            StringBuilder currentQuestion = new StringBuilder();
            boolean inQuestion = false;
            int questionCount = 1;
            
            for (String sentence : sentences) {
                sentence = sentence.trim();
                if (sentence.isEmpty()) continue;
                
                // 检查是否包含题目标志词
                if (sentence.matches(".*[问题题目][\\s]*\\d+[：:.].*") ||
                    sentence.matches(".*\\d+[、.]\\s*[A-Za-z].*") || // 包含选项的句子
                    sentence.contains("下列") && sentence.contains("正确的") ||
                    sentence.contains("以下") && sentence.contains("错误的") ||
                    sentence.contains("描述") && sentence.contains("正确的") ||
                    sentence.contains("说法") && sentence.contains("正确的") ||
                    sentence.contains("属于") && sentence.contains("是") ||
                    sentence.contains("不属于") && sentence.contains("是") ||
                    sentence.contains("特点") && sentence.contains("包括") ||
                    sentence.contains("作用") && sentence.contains("有") ||
                    sentence.contains("原理") ||
                    sentence.contains("机制") ||
                    sentence.contains("流程") ||
                    sentence.contains("步骤") ||
                    sentence.contains("方法") && sentence.contains("有") ||
                    sentence.contains("区别") && sentence.contains("与") ||
                    sentence.contains("比较") && sentence.contains("和") ||
                    sentence.contains("优势") && sentence.contains("是") ||
                    sentence.contains("缺点") && sentence.contains("是") ||
                    sentence.contains("功能") && sentence.contains("包括") ||
                    sentence.contains("分类") && sentence.contains("有") ||
                    sentence.contains("定义") ||
                    sentence.contains("概念") ||
                    sentence.contains("什么是") ||
                    sentence.contains("为什么") ||
                    sentence.contains("如何") ||
                    sentence.contains("怎样") ||
                    sentence.contains("请简述") ||
                    sentence.contains("请解释") ||
                    sentence.contains("请说明") ||
                    sentence.contains("请描述") ||
                    sentence.contains("请分析") ||
                    sentence.contains("请比较") ||
                    sentence.contains("请列举") ||
                    sentence.contains("请论述")) {
                        
                    // 如果已经在处理题目，先保存之前的题目
                    if (inQuestion && currentQuestion.length() > 20) {
                        saveQuestion(currentQuestion.toString().trim(), String.valueOf(questionCount), questions, uniqueQuestionIds);
                        questionCount++;
                    }
                    
                    // 开始新的题目
                    currentQuestion = new StringBuilder(sentence);
                    inQuestion = true;
                } else if (inQuestion && currentQuestion != null) {
                    // 继续当前题目（如果句子不是答案）
                    if (!sentence.contains("答案") && !sentence.contains("解析") && !sentence.contains("正确选项") && 
                        !sentence.contains("解题思路") && !sentence.contains("参考答案") && !sentence.contains("正确答案")) {
                        currentQuestion.append("。").append(sentence);
                    } else {
                        // 如果是答案，保存题目并开始处理答案
                        saveQuestion(currentQuestion.toString().trim(), String.valueOf(questionCount), questions, uniqueQuestionIds);
                        questionCount++;
                        currentQuestion = new StringBuilder(sentence);
                        inQuestion = false;
                    }
                }
            }
            
            // 保存最后一个题目
            if (inQuestion && currentQuestion.length() > 20) {
                saveQuestion(currentQuestion.toString().trim(), String.valueOf(questionCount), questions, uniqueQuestionIds);
            }
            
        } catch (Exception e) {
            System.out.println("使用关键词提取法时发生错误: " + e.getMessage());
        }
    }
    
    /**
     * 保存题目对象并处理去重
     */
    private void saveQuestion(String content, String number, List<Question> questions, Set<String> uniqueQuestionIds) {
        if (content == null || content.isEmpty() || questions.size() >= 30) {
            return;
        }
        
        Question question = new Question();
        question.setNumber(number);
        question.setTitle(extractQuestionTitle(content));
        question.setContent(content);
        question.setAnswer("答案未找到，请手动添加");
        
        // 使用序号和内容哈希作为唯一标识进行去重
        String questionId = number + ":" + content.hashCode();
        if (!uniqueQuestionIds.contains(questionId)) {
            uniqueQuestionIds.add(questionId);
            questions.add(question);
        }
    }
    
    /**
     * 清理HTML标签，只保留纯文本内容 - 增强版
     */
    private String cleanHtml(String html) {
        if (html == null || html.isEmpty()) {
            return "";
        }
        // 使用更强大的HTML清理方法
        String text = html.replaceAll("<[^>]*>", "").trim();
        // 移除多余的空白字符
        text = text.replaceAll("\\s+", " ");
        // 移除特殊的Unicode空白字符
        text = text.replaceAll("[\\u00A0\\u2007\\u202F]", " ");
        // 移除控制字符
        text = text.replaceAll("[\\p{Cntrl}&&[^\\r\\n\\t]]", "");
        // 移除连续的换行符
        text = text.replaceAll("\\n{3,}", "\\n\\n");
        // 移除转义字符
        text = text.replaceAll("\\\\\\\\", "\\\\");
        return text;
    }

    /**
     * 初始化SSL上下文，信任所有证书和主机名
     */
    public void initSSLContext() {
        try {
            SSLContext sslContext = SSLContext.getInstance("TLS");
            sslContext.init(null, new TrustManager[]{new TrustAllManager()}, new java.security.SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
            
            // 设置主机名验证器，接受所有主机名
            HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true);
        } catch (Exception e) {
            System.out.println("初始化SSL上下文失败: " + e.getMessage());
            e.printStackTrace();
        }
    }

    /**
     * 获取网页内容 - 增强版
     */
    public String getHtmlContent(String url) {
        try {
            URL urlObj = new URL(url);
            HttpsURLConnection connection = (HttpsURLConnection) urlObj.openConnection();
            
            // 设置请求头 - 增强版，添加更多浏览器常用请求头
            connection.setRequestMethod("GET");
            connection.setRequestProperty("User-Agent", USER_AGENTS[RANDOM.nextInt(USER_AGENTS.length)]);
            if (cookies != null && !cookies.isEmpty()) {
                connection.setRequestProperty("Cookie", cookies);
            }
            // 添加更多请求头模拟真实浏览器
            connection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
            connection.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6");
            connection.setRequestProperty("Referer", "https://www.nowcoder.com/");
            connection.setRequestProperty("Connection", "keep-alive");
            connection.setRequestProperty("Cache-Control", "max-age=0");
            connection.setRequestProperty("Upgrade-Insecure-Requests", "1");
            connection.setConnectTimeout(15000);
            connection.setReadTimeout(15000);
            connection.setInstanceFollowRedirects(true);

            // 获取响应
            int responseCode = connection.getResponseCode();
            System.out.println("请求响应码: " + responseCode);
            
            if (responseCode == HttpsURLConnection.HTTP_OK) {
                try (java.io.BufferedReader reader = new java.io.BufferedReader(
                        new java.io.InputStreamReader(connection.getInputStream(), "UTF-8"))
                ) {
                    StringBuilder content = new StringBuilder();
                    String line;
                    while ((line = reader.readLine()) != null) {
                        content.append(line).append("\n");
                    }
                    return content.toString();
                }
            } else {
                System.out.println("请求失败，响应码: " + responseCode);
                // 尝试处理重定向
                if (responseCode == HttpsURLConnection.HTTP_MOVED_PERM || 
                    responseCode == HttpsURLConnection.HTTP_MOVED_TEMP || 
                    responseCode == 307) {
                    String newUrl = connection.getHeaderField("Location");
                    if (newUrl != null) {
                        System.out.println("重定向到: " + newUrl);
                        // 递归获取重定向后的内容
                        return getHtmlContent(newUrl);
                    }
                }
                // 尝试从错误流获取内容
                try (java.io.BufferedReader reader = new java.io.BufferedReader(
                        new java.io.InputStreamReader(connection.getErrorStream(), "UTF-8"))
                ) {
                    StringBuilder errorContent = new StringBuilder();
                    String line;
                    while ((line = reader.readLine()) != null) {
                        errorContent.append(line).append("\n");
                    }
                    System.out.println("错误内容: " + errorContent.toString());
                }
            }
        } catch (IOException e) {
            System.out.println("获取网页内容时发生IO异常: " + e.getMessage());
            e.printStackTrace();
        }
        return null;
    }

    /**
     * 尝试多种选择器策略来定位题目 - 增强版
     */
    private Elements findQuestionElements(Document document) {
        // 策略1：尝试牛客网特定的题目容器选择器
        Elements elements = document.select(".question-item, .question-content, .question-stem, .topic-item, .problem-item, .test-item, .exercise-item");
        if (!elements.isEmpty()) {
            System.out.println("使用选择器1找到" + elements.size() + "个题目元素");
            return elements;
        }

        // 策略2：尝试常见的题目容器类名
        elements = document.select(".subject-item, .quiz-item, .exam-item, .lesson-item, .knowledge-item");
        if (!elements.isEmpty()) {
            System.out.println("使用选择器2找到" + elements.size() + "个题目元素");
            return elements;
        }

        // 策略3：尝试根据文本内容查找包含题目编号的元素
        elements = document.select("p:matchesOwn(^\\d+[、.]), div:matchesOwn(^\\d+[、.]), li:matchesOwn(^\\d+[、.])");
        if (!elements.isEmpty()) {
            System.out.println("使用选择器3找到" + elements.size() + "个题目元素");
            return elements;
        }

        // 策略4：尝试查找包含"问题"、"题目"关键词的元素
        elements = document.select(":matchesOwn(问题\\d+|题目\\d+|Q:[\\s]*|问:[\\s]*)");
        if (!elements.isEmpty()) {
            System.out.println("使用选择器4找到" + elements.size() + "个题目元素");
            return elements;
        }

        // 策略5：尝试查找class包含question或problem的元素
        elements = document.select("[class*=question], [class*=problem], [class*=quiz]");
        if (!elements.isEmpty()) {
            System.out.println("使用选择器5找到" + elements.size() + "个题目元素");
            return elements;
        }

        // 策略6：尝试查找包含题目样式的div
        elements = document.select("div[id*=question], div[id*=problem], div[id*=quiz]");
        if (!elements.isEmpty()) {
            System.out.println("使用选择器6找到" + elements.size() + "个题目元素");
            return elements;
        }
        
        // 策略7：尝试牛客网特定的页面结构
        elements = document.select(".nc-exam-question, .nc-question-item, .test-question-container, .exam-paper-question");
        if (!elements.isEmpty()) {
            System.out.println("使用选择器7找到" + elements.size() + "个题目元素");
            return elements;
        }
        
        // 策略8：尝试查找试卷中常见的结构
        elements = document.select(".paper-content .question, .paper-item .question, .content-item .question");
        if (!elements.isEmpty()) {
            System.out.println("使用选择器8找到" + elements.size() + "个题目元素");
            return elements;
        }

        System.out.println("所有选择器策略都未找到题目元素");
        return elements;
    }

    /**
     * 解析单个题目 - 增强版
     */
    private Question parseQuestion(Element element, Document document) {
        Question question = new Question();

        try {
            // 提取题目文本
            String questionText = element.text();
            if (questionText.isEmpty() || questionText.length() < 10) {
                // 尝试获取元素的HTML内容，因为有些内容可能在子元素中
                questionText = cleanHtml(element.html());
                if (questionText.isEmpty() || questionText.length() < 10) {
                    return null;
                }
            }

            // 提取题目编号
            String number = extractQuestionNumber(questionText);
            if (number != null) {
                question.setNumber(number);
            } else {
                // 如果没有明确的编号，尝试从元素ID或属性中提取
                String id = element.id();
                if (id != null && id.matches(".*\\d+.*")) {
                    Pattern idPattern = Pattern.compile("\\d+");
                    Matcher idMatcher = idPattern.matcher(id);
                    if (idMatcher.find()) {
                        question.setNumber(idMatcher.group());
                    }
                }
            }

            // 提取题目标题
            String title = extractQuestionTitle(questionText);
            if (title != null) {
                question.setTitle(title);
            } else {
                // 如果没有明确的标题，使用内容的前60个字符作为标题
                question.setTitle(questionText.length() > 60 ? questionText.substring(0, 60) + "..." : questionText);
            }

            // 设置题目内容
            question.setContent(questionText);

            // 尝试提取答案 - 增强版
            String answer = extractAnswer(element, document);
            question.setAnswer(answer);

        } catch (Exception e) {
            System.out.println("解析题目时发生错误: " + e.getMessage());
        }

        return question;
    }

    /**
     * 尝试提取答案 - 增强版
     */
    private String extractAnswer(Element questionElement, Document document) {
        StringBuilder answerBuilder = new StringBuilder();

        try {
            // 策略1：查找题目元素后面的答案元素
            Element nextElement = questionElement.nextElementSibling();
            if (nextElement != null && (nextElement.text().contains("答案") || nextElement.text().contains("解析"))) {
                answerBuilder.append(nextElement.text());
                return answerBuilder.toString();
            }

            // 策略2：查找题目元素的兄弟元素中包含答案关键词的元素
            Elements siblings = questionElement.siblingElements();
            for (Element sibling : siblings) {
                String siblingText = sibling.text();
                if (siblingText.contains("答案") || siblingText.contains("解析") || 
                    siblingText.contains("详解") || siblingText.contains("正确选项")) {
                    // 检查元素与题目元素的距离
                    if (Math.abs(sibling.elementSiblingIndex() - questionElement.elementSiblingIndex()) < 10) {
                        answerBuilder.append(siblingText).append("\n");
                    }
                }
            }
            if (answerBuilder.length() > 0) {
                return answerBuilder.toString().trim();
            }

            // 策略3：查找页面中包含"答案"、"解析"关键词的元素，并按位置排序
            Elements answerElements = document.select(":matchesOwn(答案|解析|参考答案|详解|正确选项)");
            if (!answerElements.isEmpty()) {
                // 按与题目元素的距离排序
                answerElements.sort((e1, e2) -> {
                    int dist1 = Math.abs(e1.elementSiblingIndex() - questionElement.elementSiblingIndex());
                    int dist2 = Math.abs(e2.elementSiblingIndex() - questionElement.elementSiblingIndex());
                    return Integer.compare(dist1, dist2);
                });
                
                // 取最近的3个答案元素
                int count = 0;
                for (Element answerElement : answerElements) {
                    answerBuilder.append(answerElement.text()).append("\n");
                    if (++count >= 3) break;
                }
                if (answerBuilder.length() > 0) {
                    return answerBuilder.toString().trim();
                }
            }

            // 策略4：查找特定的答案容器 - 增强版，增加更多选择器
            String[] answerSelectors = {
                ".answer-container", ".analysis-container", ".reference-answer", ".answer-explanation",
                ".solution", ".explanation", ".answer-detail", ".analysis-detail", ".correct-answer",
                ".answer-content", ".analysis-content", ".answer-block", ".analysis-block",
                ".nc-answer", ".nc-analysis", ".exam-answer", ".test-answer", ".question-answer",
                ".answer-section", ".analysis-section", ".answer-part", ".analysis-part"
            };
            
            for (String selector : answerSelectors) {
                Elements answerContainers = document.select(selector);
                if (!answerContainers.isEmpty()) {
                    for (Element container : answerContainers) {
                        answerBuilder.append(cleanHtml(container.html())).append("\n");
                    }
                    if (answerBuilder.length() > 0) {
                        return answerBuilder.toString().trim();
                    }
                    break;
                }
            }

            // 策略5：尝试查找选项和正确答案
            Elements optionElements = document.select(".option, .answer-option, .question-option, .nc-option");
            if (!optionElements.isEmpty()) {
                StringBuilder optionsBuilder = new StringBuilder();
                for (Element option : optionElements) {
                    String optionText = cleanHtml(option.html());
                    if (!optionText.isEmpty()) {
                        optionsBuilder.append(optionText).append("\n");
                    }
                }
                
                String options = optionsBuilder.toString().trim();
                if (!options.isEmpty()) {
                    // 检查是否有正确答案标记
                    Elements correctOptions = document.select(".option.correct, .correct-answer, .right-answer, .correct-option, .right-option, .nc-correct");
                    if (!correctOptions.isEmpty()) {
                        String correctAnswer = cleanHtml(correctOptions.text());
                        answerBuilder.append("选项：\n").append(options).append("\n\n正确答案：").append(correctAnswer);
                    } else {
                        answerBuilder.append("选项：\n").append(options).append("\n\n正确答案未明确标记");
                    }
                }
            }

            // 策略6：尝试查找class或id包含answer的元素
            Elements generalAnswerElements = document.select("[class*=answer], [id*=answer], [class*=analysis], [id*=analysis]");
            if (!generalAnswerElements.isEmpty()) {
                for (Element elem : generalAnswerElements) {
                    String elemText = cleanHtml(elem.text());
                    if (elemText.length() > 20 && !elemText.contains("题目") && !elemText.contains("问题")) {
                        answerBuilder.append(elemText).append("\n");
                    }
                }
                if (answerBuilder.length() > 0) {
                    return answerBuilder.toString().trim();
                }
            }

        } catch (Exception e) {
            System.out.println("提取答案时发生错误: " + e.getMessage());
        }

        // 如果仍然没有找到答案，标记为"答案未找到，请手动添加"
        if (answerBuilder.length() == 0) {
            answerBuilder.append("答案未找到，请手动添加");
        }

        return answerBuilder.toString();
    }

    /**
     * 尝试从文本中提取题目编号
     */
    private String extractQuestionNumber(String text) {
        // 匹配"1、" "问题1:" "题目1" 等格式
        java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("^(\\d+)[、.\\s]*");
        java.util.regex.Matcher matcher = pattern.matcher(text);
        if (matcher.find()) {
            return matcher.group(1);
        }

        // 匹配"问题1:" "题目1:" 等格式
        pattern = java.util.regex.Pattern.compile("[问题题目](\\d+)[：:.]?");
        matcher = pattern.matcher(text);
        if (matcher.find()) {
            return matcher.group(1);
        }

        // 匹配"Q1:" "第1题:" 等格式
        pattern = java.util.regex.Pattern.compile("Q(\\d+)[：:.]?|第(\\d+)题[:：]?");
        matcher = pattern.matcher(text);
        if (matcher.find()) {
            return matcher.group(1) != null ? matcher.group(1) : matcher.group(2);
        }

        return null;
    }

    /**
     * 尝试从文本中提取题目标题
     */
    private String extractQuestionTitle(String text) {
        // 匹配"1、标题" "问题1:标题" 等格式
        java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("^(?:\\d+[、.\\s]*|[问题题目]\\d+[：:.]?)\\s*([^\\n]+?)(?=\\n|$|\\s*[\\(（][参考答案|解析]\\))");
        java.util.regex.Matcher matcher = pattern.matcher(text);
        if (matcher.find()) {
            return matcher.group(1).trim();
        }

        // 匹配"Q:标题" "问:标题" 等格式
        pattern = java.util.regex.Pattern.compile("^(?:Q[:：]|问[:：])\\s*([^\\n]+?)(?=\\n|$)");
        matcher = pattern.matcher(text);
        if (matcher.find()) {
            return matcher.group(1).trim();
        }

        // 如果没有明确的标题，就返回null
        return null;
    }

    /**
     * 信任所有SSL证书的TrustManager实现
     */
    private static class TrustAllManager implements TrustManager, javax.net.ssl.X509TrustManager {
        @Override
        public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) {}

        @Override
        public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) {}

        @Override
        public java.security.cert.X509Certificate[] getAcceptedIssuers() {
            return new java.security.cert.X509Certificate[0];
        }
    }
}